# -*- coding:utf-8 -*-
# @FileName  :lda_demo.py
# @Time      :2024/11/19 08:22
# @Author    :lin

import os
import jieba
import pandas as pd
# 读取数据
file_path = "上海旅游景点.xlsx"
df = pd.read_excel(file_path,)
specified_column = df['关键词1']
print(specified_column)
# 分词
word_list = [jieba.lcut(i) for i in specified_column]
print(word_list)

with open('stopwords.txt', 'r', encoding='utf-8') as f:
    stop_words = set([line.strip() for line in f.readlines()])

for i in range(len(word_list)):
    word_list[i] = [word for word in word_list[i] if word not in stop_words]

import gensim
import gensim.corpora as corpora
from gensim.models import LdaModel
from gensim.models import CoherenceModel
import pyLDAvis
import pyLDAvis.gensim as gensimvis
from pprint import pprint

#预处理：分词、去停用词，上面已经完成

# 创建词典
dictionary = corpora.Dictionary(word_list)
print("字典长度为：", len(dictionary), "\n", dictionary)
# 构建语料库
corpus = [dictionary.doc2bow(text) for text in word_list]
print("语料库长度为：", len(corpus), "\n", corpus)
# 训练LDA模型
# lda_model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=3)
# # 使用pyLDAvis可视化
# lda_display = gensimvis.prepare(lda_model, corpus, dictionary, sort_topics=False)
# pyLDAvis.display(lda_display)
# pyLDAvis.save_html(lda_display, 'lda_vis.html') # 保存为html文件

# 准备一个空列表用于存储主题数和对应的模型
num_topics_range = range(2, 5)
perplexities = []
coherences = []

for num_topics in num_topics_range:
    # 训练LDA模型
    lda_model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics)
    # 计算困惑度
    perplexity = lda_model.log_perplexity(corpus)
    perplexities.append(perplexity)
    print(f"Num topics:{num_topics}, Perplexity: {perplexity}")
    # 计算主题一致性
    coherence_model_lda = CoherenceModel(model=lda_model, texts=word_list, dictionary=dictionary, coherence='c_v')
    coherence_lda = coherence_model_lda.get_coherence()
    coherences.append(coherence_lda)
    print(f"Num topics:{num_topics}, Coherence Score: {coherence_lda}")


